{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# W3_冯炳驹_124298228"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第二步：调整树的参数：max_depth & min_child_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# path to where the data lies\n",
    "dpath = './data/'\n",
    "train = pd.read_csv(dpath +\"RentListingInquries_FE_train.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEFCAYAAAD0cwBnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGmhJREFUeJzt3XuUXWWd5vFvpRJyIZV0jRQyjiCGdh6ZFjQGSGySMUIU\nwQtqd9vKZdHShkvTRLvphpYkKq6wWKLBTnBMnEAgELRZRhA7DhBahFymCRC5JAv8CRkv9IzaJeRS\nGJIQcuaPdx9zOqTO2ZvUqXNOneezVhZnv+fdu36brFVP3v3u/e6OUqmEmZlZXsMaXYCZmbUWB4eZ\nmRXi4DAzs0IcHGZmVoiDw8zMChne6ALqrbe3z7eNmZkV1NPT1dHfdx5xmJlZIQ4OMzMrxMFhZmaF\n1G2OQ1InsAQQUAIuAkYAK4Fnsm6LIuJ2STOBC4E9wLyIWClpNLAcOBzoA86LiF5JU4AFWd9VEXFV\nvc7BzMxerZ4jjg8BRMTJwBzgamAScF1ETM/+3C7pCGAWcDJwGnCNpJHAxcDGiJgG3JIdA2AxcBYw\nFZgsaWIdz8HMzPZTtxFHRHxP0sps803AVlJwSNKZpFHHZ4GTgHURsQvYJelZ4HhSMFyb7X83MFfS\nOGBkRGwmHeheYAbwWH91dHePYfjwzgE/PzOzdlXX23EjYo+kZcBHgT8F/gtwQ0RskDQb+ALwOLCt\nYrc+YDwwrqK9sm37fn0nVKthy5YdA3AmZmbtpaenq9/v6j45HhHnAf+VNN+xKiI2ZF/dCUwkBUFl\nhV2k0Ull+4HaKtvNzGyQ1C04JJ0r6XPZ5g5gL3CHpJOytlOBDcDDwDRJoySNB44FNgHrgDOyvqcD\nayJiO7Bb0jGSOkhzImvqdQ5mZvZq9bxUdQdwk6TVpLupPgs8B1wv6WXg18AFEbFd0kJSAAwDZkfE\nTkmLgGWS1gK7SRPikO7Oug3oJI1g1tfxHMzMbD8dQ/1FTl5yxKw1PHLZrEaXMOSdOH9h7r5ecsTM\nzAaMg8PMzApxcJiZWSEODjMzK8TBYWZmhTg4zMysEAeHmZkV4uAwM7NCHBxmZlaIg8PMzApxcJiZ\nWSEODjMzK8TBYWZmhTg4zMysEAeHmZkV4uAwM7NCHBxmZlaIg8PMzApxcJiZWSEODjMzK8TBYWZm\nhTg4zMyskOH1OrCkTmAJIKAEXATsBG7OtjcBl0TEXkkzgQuBPcC8iFgpaTSwHDgc6APOi4heSVOA\nBVnfVRFxVb3OwczMXq2eI44PAUTEycAc4GrgOmBOREwDOoAzJR0BzAJOBk4DrpE0ErgY2Jj1vSU7\nBsBi4CxgKjBZ0sQ6noOZme2nbsEREd8DLsg23wRsBSYBD2ZtdwMzgJOAdRGxKyK2Ac8Cx5OC4Z7K\nvpLGASMjYnNElIB7s2OYmdkgqdulKoCI2CNpGfBR4E+B92a/8CFdfhoPjAO2Vex2oPbKtu379Z1Q\nrYbu7jEMH955kGdiZtb6enq6BuQ4dQ0OgIg4T9IVwHpgdMVXXaRRyPbsc7X2Wn37tWXLjoMp38xs\nyOjt7cvdt1rI1O1SlaRzJX0u29wB7AUelTQ9azsdWAM8DEyTNErSeOBY0sT5OuCMyr4RsR3YLekY\nSR2kOZE19ToHMzN7tXqOOO4AbpK0GhgBfBZ4Glgi6ZDs84qIeEXSQlIADANmR8ROSYuAZZLWArtJ\nE+KQ7s66Degk3VW1vo7nYGZm++kolUq1e7Ww3t6+oX2CZkPEI5fNanQJQ96J8xfm7tvT09XR33d+\nANDMzApxcJiZWSEODjMzK8TBYWZmhTg4zMysEAeHmZkV4uAwM7NCHBxmZlaIg8PMzApxcJiZWSEO\nDjMzKyRXcGSLEiLpDyV9QJIDx8ysTdUMAEmfB26QdBSwGvgb4Jv1LszMzJpTnpHDh4GZpGXNl0fE\nDMDv+TYza1N5gqMzInYBHwT+V3aZ6tD6lmVmZs0qT3D8UNIm4BDSpaoHge/XtSozM2taNYMjIv6O\n9ArXKRGxF7g0Iq6oe2VmZtaU8kyOdwNzgX+R9DpgVtZmZmZtKM+lqiXAI8DrgD7gV8DyehZlZmbN\nK09wvDki/iewNyJ2R8Rs4I11rsvMzJpUnuDYI2k8UAKQ9BZgb12rMjOzpjU8R58vAA8AR0n6HvAu\n4Px6FmVmZs2rZnBExD2SHgUmA53AhRHxm2r7SBoBLAWOBkYC84DngJXAM1m3RRFxu6SZwIXAHmBe\nRKyUNJo0j3I4aV7lvIjolTQFWJD1XRURVxU9YTMzOzh57qp6D3BXRPwACOBfJf1xjd3OAZ6PiGnA\n+4GvA5OA6yJievbndklHALOAk4HTgGskjQQuBjZm+98CzMmOu5j0BPtUYLIkP8FuZjbI8sxxzCeN\nCIiIID3TsaDGPt8h3cIL0EEaIUwCPiBptaQbJXUBJwHrImJXRGwDngWOJwXDPdn+dwMzJI0DRkbE\n5ogoAfcCM3Kep5mZDZA8cxyjImJTeSMifpJdiupXRLwIkIXDCtKIYSRwQ0RskDSbNHfyOLCtYtc+\nYDwwrqK9sm37fn0n1Cq+u3sMw4d31upmZjbk9fR0Dchx8gTHTyR9Gbg12/4E8NNaO0k6ErgT+EZE\nfEvSH0TE1uzrO4HrSUuYVJ5JF7CVFBBdVdoq26vasmVHrS5mZm2ht7cvd99qIZPnUtVfAmOBb5Pm\nG8aSVsvtl6TXA6uAKyJiadZ8r6STss+nAhuAh4FpkkZlt/weC2wC1pEuiQGcDqyJiO3AbknHSOog\nzYmsyVG/mZkNoDx3VW0BLil43CuBbmCupPJcx98CX5P0MvBr4IKI2C5pISkAhgGzI2KnpEXAMklr\ngd2kCXGAi4DbSHd3rYqI9QXrMjOzg9RRKpWqdpD0F8BXSUEAabK7FBEtMXHQ29tX/QTNrCk8ctms\nRpcw5J04f2Huvj09XR39fZdnjuPzwPTKCXIzM2tfeeY4/q9Dw8zMyvKMODZIWkGa7N5ZboyIW+pW\nlZmZNa08wTGe9MzEuyraSqQ7rMzMrM3kuavqU5Be6JTdYWVmZm2sZnBIejtwOzAmW2RwNfDxiPhx\nvYszM7Pmk2dy/Hrgo6RFC/8faQHCxXWtyszMmlae4BgTEU+XNyLiPtK6U2Zm1obyBMcL2eWq8hsA\nzwZeqGtVZmbWtPLcVXUxsAz4I0lbSS9iOruuVZmZWdPKExzvjYipkg4FOrPFBs3MrE3lCY6/BhZH\nxO/qXYyZmTW/PMHxnKT7gfXAS+XGiPhS3aoyM7OmlSc4Hqr43O9qiWZm1h7yBMfR5afHzczM8tyO\n+zZJY+teiZmZtYQ8I469wC8lBf9xjuOUulVlZmZNK09wXF73KszMrGXkCQ6/etXMzH4vT3BcVfF5\nBHA8sIa0Sq6ZmbWZPO/jeE/ltqQ3A1+rW0VmZtbU8txV9R9ExM+At9ahFjMzawF5XuR0E/vmOTqA\nY4FNNfYZASwFjiYtwT4PeAq4OTvWJuCSiNgraSZwIbAHmBcRKyWNBpYDh5NeW3teRPRmL5JakPVd\nFRFXYWZmgyrPiOMB4MHsz4+ALwFn1djnHNKLn6YB7we+DlwHzMnaOoAzJR0BzAJOBk4DrpE0krQi\n78as7y3AnOy4i7OfPRWYLGlizvM0M7MBkic47gDGRsQy4IfAFGqPVL4DzM0+d5BGCJNI4QNwNzAD\nOAlYFxG7ImIb8Cxp8n0qcE9lX0njgJERsTkiSsC92THMzGwQ5bmr6jbgyexzHylsbgX+pL8dIuJF\nAEldwArSiOGr2S/88nHGA+OAbRW7Hqi9sm37fn0n1Cq+u3sMw4d31upmZjbk9fR0Dchx8gTHmyLi\nwwDZuzjmSHq81k6SjgTuBL4REd+SdG3F113AVlIQdNVor9W3qi1bdtTqYmbWFnp7+3L3rRYyeS5V\nlSQdV96Q9Fbg5Wo7SHo9sAq4IiKWZs2PSZqefT6d9CzIw8A0SaMkjWffxPs64IzKvllo7ZZ0jKQO\n0pzImhz1m5nZAMoz4vg74D5J/0aarziMNPldzZVANzBXUnmu4zPAQkmHAE8DKyLiFUkLSQEwDJgd\nETslLQKWSVoL7GbfZPxFpEtnnaS7qtbnPVEzMxsYHaVS7RVFsl/2x5FGGptb6W2Avb19XjLFrAU8\nctmsRpcw5J04f2Huvj09Xf2+f6nmpSpJHwc2RMQG4HfATySdmfunm5nZkJJnjmMO2W2vEbGZdFut\nH7wzM2tTeYLjkIj4TXkjIv4dv0LWzKxt5ZkcXyvp26RJaYCPA/9av5LMzKyZ5QmOS4BLSetJ7Sbd\nAfWNehZlZmbNK8+y6rsk3QPsIt0G+0BE7K57ZWZm1pTy3FV1LnAXaaXbNwF3SDq/znWZmVmTynOp\n6jLgpIh4HkDS1aQVc5dW28nMzIamPHdVdZZDAyAifgvsrV9JZmbWzPKMOJ6Q9I/Ajdn2XwJP1K8k\nMzNrZnlGHDNJE+NLSW/w2w38VR1rMjOzJpbnrqqXgCsGoRYzM2sBeUYcZmZmv9dvcEg6dDALMTOz\n1lBtxPEAgCQ/JW5mZr9XbY5jrKTlwPsljdr/y4jwQ4BmZm2oWnC8D3gPMA14cHDKMTOzZtdvcETE\nc8Atkp4AngKU9d8UEXsGqT4zM2syee6qGgE8AywDbgJ+KWlyXasyM7OmlefJ8QXAn0fEegBJU4Dr\ngZPqWZiZmTWnPCOOseXQAIiIh4BXTZabmVl7yBMcL0g6s7wh6SPA81X6m5nZEJbnUtUFwHJJN5Le\nNb4ZOKeuVZmZWdPKs1bVM8Dk7EnyYRHRl/fg2ST6lyNiuqSJwErSRDvAooi4XdJM0mtp9wDzImKl\npNHAcuBwoA84LyJ6s/mVBVnfVRFxVf5TNTOzgZBnxAFARPyuyIElXQ6cC5T3mwRcFxHzK/ocAcwC\nTiDNm6yVdB9wMbAxIr4o6RPAHOAzwGLgT4D/A/xA0sSIeKxIXWZmdnByB8drsBn4GHBrtj0JUDZf\n8gzwWdKdWesiYhewS9KzwPHAVODabL+7gbmSxgEjI2Iz6UD3AjOAqsHR3T2G4cM7B/TEzMxaUU9P\n14Acp2ZwSLooIhYXPXBEfFfS0RVNDwM3RMQGSbOBLwCPA9sq+vQB44FxFe2Vbdv36zuhVh1btuwo\nWrqZ2ZDU25t7pqFqyOS5q+qvc/+k6u6MiA3lz8BEUhBUVtcFbN2v/UBtle1mZjaI8lyqek7S/cB6\n4KVyY0R8qeDPulfSpRHxMHAqsIE0Crk6W0RxJHAssAlYB5yRfX86sCYitkvaLekY0hzHaYAnx83M\nBlme4Hio4nPHQfysi4HrJb0M/Bq4IAuDhcAa0uhndkTslLQIWCZpLelVtWdlx7gIuA3oJN1Vtf5V\nP8XMzOqqo1Qq1eyU3Yp7DGk0MLroHVaN1NvbV/sEzazhHrlsVqNLGPJOnL8wd9+enq5+Bwo15zgk\nnQI8AdwFvB74uaT35f7pZmY2pOSZHL+GdHvs1oj4FfBu4Ct1rcrMzJpWnuAYFhG/Lm9ExFN1rMfM\nzJpcnsnxf5P0QaAk6Q+AS4Bf1rcsMzNrVnlGHBcCZwNHkm6DfQdp4UMzM2tDeRY5/Hfgk9mSHy9H\nxEu19jEzs6Erz5Ijx5FeG3tUtv0T0mq1m+tcm5mZNaE8l6oWkx7MOywiDgPmA0vrW5aZmTWrPMEx\nOiLuLm9ExJ2kBQfNzKwN9XupStJR2ccnJP0DcCPpBUpnk5YIMTOzNlRtjuNBoERan2o66e6qshLp\nBUxmZtZm+g2OiHjzYBZiZmatIc9dVSI9t9Fd2R4R59erKDMza155nhy/E/gn4Mk612JmZi0gT3Bs\nfQ0vbTIzsyEqT3DcLOlq4Ieku6oAiIjVdavKzMyaVp7gmA6cCPxxRVsJOKUeBZmZWXPLExwnRMRb\n6l6J2UH6+5VzGl3CkPeVD85rdAnWBPI8Ob5R0vF1r8TMzFpCnhHHBOAxSb8CdpMeCCxFxIS6VmZm\nZk0pT3B8pO5VmJlZy8gTHO/up/2WgSzEzMxaQ57geE/F5xHANGA1OYJD0mTgyxExXdIfAjeT7sja\nBFwSEXslzSStg7UHmBcRKyWNBpYDhwN9pPd/9EqaAizI+q6KiKtynqeZmQ2QmpPjEfGpij/nABOB\nI2rtJ+ly4AZgVNZ0HTAnIqaR5knOlHQEabHEk4HTgGskjQQuBjZmfW8ByrfLLAbOAqYCkyVNzH+q\nZmY2EPKMOPb3InB0jn6bgY8Bt2bbk0gr7gLcDbwPeAVYFxG7gF2SngWOJwXDtRV952avrh1ZfvOg\npHuBGcBj1Yro7h7D8OGd+c7MzKrq6elqdAl2EAbq7y/PIoc/Il1egjRSmAD8oNZ+EfFdSUdXNHVE\nRPk4fcB40guhtlX0OVB7Zdv2/frWvLNry5YdtbqYWU69vX2NLsEOQpG/v2ohk2fE8cWKzyXgtxHx\nVO6fvs/eis9dwFZSEHTVaK/V18zMBlG/cxySjsreAvizij8/B16seDtgEY9Jmp59Pp30FsGHgWmS\nRkkaDxxLmjhfB5xR2TcitgO7JR0jqYM0J+I3EZqZDbK8bwAsKwFvIN1dVXTi4DJgiaRDgKeBFRHx\niqSFpAAYBsyOiJ2SFgHLJK0lPXR4VnaMi4Dbsp+9KiLWF6zBzMwOUu43AEoaC8wn/Ut/Zp6DR8TP\ngSnZ559ygGdCImIJsGS/th3Anx2g70Pl45mZWWPkWasKSaey70VOx0XEffUryczMmlnVyXFJh5Ke\nvzgNmOnAMDOzapPjpwIbs823OTTMzAyqjzjuA14mPaj3pKRyu1fHNTNrY9WC481VvjMzszZV7a6q\nXwxmIWZm1hpy3VVlZmZW5uAwM7NCHBxmZlaIg8PMzApxcJiZWSEODjMzK8TBYWZmhTg4zMysEAeH\nmZkV4uAwM7NCHBxmZlaIg8PMzApxcJiZWSEODjMzK8TBYWZmhTg4zMyskGpvAKwLST8GtmebPwOu\nBm4GSsAm4JKI2CtpJnAhsAeYFxErJY0GlgOHA33AeRHRO8inYGbW1gZ1xCFpFNAREdOzP58CrgPm\nRMQ00vvMz5R0BDALOBk4DbhG0kjgYmBj1vcWYM5g1m9mZoM/4ng7MEbSquxnXwlMAh7Mvr8beB/w\nCrAuInYBuyQ9CxwPTAWureg7dxBrNzMzBj84dgBfBW4A3kL65d8REaXs+z5gPDAO2Fax34Hay21V\ndXePYfjwzgEp3qzd9fR0NboEOwgD9fc32MHxU+DZLCh+Kul50oijrAvYSpoD6arRXm6rasuWHQNQ\ntpkB9Pb2NboEOwhF/v6qhcxg31V1PjAfQNIbSCOIVZKmZ9+fDqwBHgamSRolaTxwLGnifB1wxn59\nzcxsEA32iONG4GZJa0l3UZ0P/BZYIukQ4GlgRUS8ImkhKRiGAbMjYqekRcCybP/dwFmDXL+ZWdsb\n1OCIiP5+2b/7AH2XAEv2a9sB/Fl9qjMzszz8AKCZmRXi4DAzs0IG/cnxZvaZr3y/0SW0hQV//+FG\nl2BmB8EjDjMzK8TBYWZmhTg4zMysEAeHmZkV4uAwM7NCHBxmZlaIg8PMzApxcJiZWSEODjMzK8TB\nYWZmhTg4zMysEAeHmZkV4uAwM7NCHBxmZlaIg8PMzApxcJiZWSEODjMzK8TBYWZmhTg4zMyskJZ7\n57ikYcA3gLcDu4BPR8Szja3KzKx9tOKI4yPAqIh4F/APwPwG12Nm1lZaMTimAvcARMRDwAmNLcfM\nrL10lEqlRtdQiKQbgO9GxN3Z9i+BCRGxp7GVmZm1h1YccWwHuiq2hzk0zMwGTysGxzrgDABJU4CN\njS3HzKy9tNxdVcCdwHsl/W+gA/hUg+sxM2srLTfHYWZmjdWKl6rMzKyBHBxmZlaIg8PMzAppxclx\nw0uvDAWSJgNfjojpja7F8pM0AlgKHA2MBOZFxPcbWtQg84ijdXnplRYm6XLgBmBUo2uxws4Bno+I\nacD7ga83uJ5B5+BoXV56pbVtBj7W6CLsNfkOMDf73AG03QPIDo7WNQ7YVrH9iiRfemwREfFd4OVG\n12HFRcSLEdEnqQtYAcxpdE2DzcHRurz0ilmDSDoS+BFwa0R8q9H1DDYHR+vy0itmDSDp9cAq4IqI\nWNroehrBlzZal5deMWuMK4FuYK6k8lzH6RHxUgNrGlRecsTMzArxpSozMyvEwWFmZoU4OMzMrBAH\nh5mZFeLgMDOzQhwc1tYknSDphirff0jS39a5hh/l6PNzSUcP4M+8WdJfDNTxrL34OQ5raxHxKPDp\nKl0mDUIZ0wfhZ5gNGAeHtTVJ04EvZpsPA9OAHuBS4BfARVm/X5AWt/sfwNuATtKS6N/O/uV+HnAY\n8M/AAuCbwJHAXuBzEfEvkk4FrgVKwBbgk8Dns+Ovj4jJOertBL5CCptO4OaI+JqkO4BvRcSKrN+j\nwAWkpWkWAa8DdgCXRsRjxf9Pme3jS1Vm+xySLVP/N6R3LDwFLAYWR8RNpMXsNkTEJOC/A7MlTcj2\nfSMwMSKuJAXH0qzfh4FvZgvizQEuiogTSAHzzoiYBZAnNDIzs/7vBE4CzpQ0DbgV+ASApLcAoyPi\nx8Ay4PKs/wXAP73W/zlmZR5xmO1zT/bfTcB/OsD3M4Axks7Ptg8F/ij7/OOKRSZnAG+V9KVsewRw\nDPB94E5J3wPuioj7XkONM4B3SDol2x4LHEd6t8f1WUB9ErhN0ljgROAmSeX9x0p63Wv4uWa/5+Aw\n22dn9t8Saf2v/XUC52T/ki8vdvcCcDbw0n79TomIF7J+bwB+ExGPS/pn4IPAtZJWRMTVBWvsJI0g\n7siOfRjwu4jYLWklaYTzceADWd+dEfGO8s6S3pjVbPaa+VKVWXV72PcPrPuBiwEk/WfgSeCoA+xz\nP/BXWb//lvUbI2k90BUR/wh8DXhn1r/Iu1TuB2ZKGpGNKNYC5ctctwKXAS9ExC8iYhvwjKRzslre\nC6zO+XPM+uXgMKtuNXC2pEuBq4DRkjaRfoFfHhGbD7DPpcAUSU8CtwPnRkQfaVXVmyVtIM03fCHr\nfxfwhKQ8r5FdDDwDPAY8CtwUEQ8ARMQ6YDywvKL/2cCns1quAf48IryyqR0Ur45rZmaFeI7DrElk\nDwJ2H+CrxRGxeLDrMeuPRxxmZlaI5zjMzKwQB4eZmRXi4DAzs0IcHGZmVoiDw8zMCvn/IYm+TmHe\nrR4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x273e6488a20>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(train.interest_level);\n",
    "pyplot.xlabel('interest_level');\n",
    "pyplot.ylabel('Number of occurrences');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# drop ids and get labels\n",
    "y_train = train['interest_level']\n",
    "\n",
    "train = train.drop([\"interest_level\"], axis=1)\n",
    "X_train = np.array(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': range(3, 10, 2), 'min_child_weight': range(1, 6, 2)}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "max_depth = range(3,10,2)\n",
    "min_child_weight = range(1,6,2)\n",
    "param_test2_1 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 默认参数，此时学习率为0.1，比较大，观察弱分类数目的大致范围 （采用默认参数配置，看看模型是过拟合还是欠拟合）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#直接调用xgboost内嵌的交叉验证（cv），可对连续的n_estimators参数进行快速交叉验证\n",
    "#而GridSearchCV只能对有限个参数进行交叉验证\n",
    "def modelfit(alg, X_train, y_train, cv_folds=None, early_stopping_rounds=10):\n",
    "    xgb_param = alg.get_xgb_params()\n",
    "    xgb_param['num_class'] = 9\n",
    "    \n",
    "    #直接调用xgboost，而非sklarn的wrapper类\n",
    "    xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "        \n",
    "    cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "             metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "  \n",
    "    cvresult.to_csv('1_nestimators.csv', index_label = 'n_estimators')\n",
    "    \n",
    "    #最佳参数n_estimators\n",
    "    n_estimators = cvresult.shape[0]\n",
    "    \n",
    "    # 采用交叉验证得到的最佳参数n_estimators，训练模型\n",
    "    alg.set_params(n_estimators = n_estimators)\n",
    "    alg.fit(X_train, y_train, eval_metric='mlogloss')\n",
    "        \n",
    "    #Predict training set:\n",
    "    #train_predprob = alg.predict_proba(X_train)\n",
    "    #logloss = log_loss(y_train, train_predprob)\n",
    "\n",
    "   #Print model report:\n",
    "   # print (\"logloss of train :\" )\n",
    "   # print logloss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': range(3, 10, 2), 'min_child_weight': range(1, 6, 2)}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "max_depth = range(3,10,2)\n",
    "min_child_weight = range(1,6,2)\n",
    "param_test2_1 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda2\\envs\\python3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:667: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59843, std: 0.00175, params: {'min_child_weight': 1, 'max_depth': 3},\n",
       "  mean: -0.59834, std: 0.00193, params: {'min_child_weight': 3, 'max_depth': 3},\n",
       "  mean: -0.59847, std: 0.00214, params: {'min_child_weight': 5, 'max_depth': 3},\n",
       "  mean: -0.58638, std: 0.00190, params: {'min_child_weight': 1, 'max_depth': 5},\n",
       "  mean: -0.58717, std: 0.00228, params: {'min_child_weight': 3, 'max_depth': 5},\n",
       "  mean: -0.58750, std: 0.00187, params: {'min_child_weight': 5, 'max_depth': 5},\n",
       "  mean: -0.58740, std: 0.00294, params: {'min_child_weight': 1, 'max_depth': 7},\n",
       "  mean: -0.58671, std: 0.00230, params: {'min_child_weight': 3, 'max_depth': 7},\n",
       "  mean: -0.58737, std: 0.00286, params: {'min_child_weight': 5, 'max_depth': 7},\n",
       "  mean: -0.60047, std: 0.00427, params: {'min_child_weight': 1, 'max_depth': 9},\n",
       "  mean: -0.59636, std: 0.00389, params: {'min_child_weight': 3, 'max_depth': 9},\n",
       "  mean: -0.59499, std: 0.00287, params: {'min_child_weight': 5, 'max_depth': 9}],\n",
       " {'max_depth': 5, 'min_child_weight': 1},\n",
       " -0.58638433769908205)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=236,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_test2_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_1.grid_scores_, gsearch2_1.best_params_,     gsearch2_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([  229.33192143,   210.90415149,   178.48019333,   256.28937631,\n",
       "          256.74911108,   258.99135313,   329.00064907,   324.84363189,\n",
       "          319.41760831,   396.71612206,   389.68874869,  1461.11516743]),\n",
       " 'mean_score_time': array([ 0.38103004,  0.3748302 ,  0.36962495,  0.64098263,  0.63756652,\n",
       "         0.65086613,  1.05732589,  1.05383363,  1.07205033,  2.15546632,\n",
       "         2.72363644,  1.420432  ]),\n",
       " 'mean_test_score': array([-0.5984267 , -0.59833602, -0.59847289, -0.58638434, -0.58717226,\n",
       "        -0.58749989, -0.58739564, -0.5867104 , -0.58736753, -0.60047428,\n",
       "        -0.59635735, -0.59499414]),\n",
       " 'mean_train_score': array([-0.57486472, -0.57554839, -0.57580899, -0.50677784, -0.51245121,\n",
       "        -0.51654178, -0.40360718, -0.42529713, -0.43898496, -0.27728054,\n",
       "        -0.32429287, -0.35287173]),\n",
       " 'param_max_depth': masked_array(data = [3 3 3 5 5 5 7 7 7 9 9 9],\n",
       "              mask = [False False False False False False False False False False False False],\n",
       "        fill_value = ?),\n",
       " 'param_min_child_weight': masked_array(data = [1 3 5 1 3 5 1 3 5 1 3 5],\n",
       "              mask = [False False False False False False False False False False False False],\n",
       "        fill_value = ?),\n",
       " 'params': ({'max_depth': 3, 'min_child_weight': 1},\n",
       "  {'max_depth': 3, 'min_child_weight': 3},\n",
       "  {'max_depth': 3, 'min_child_weight': 5},\n",
       "  {'max_depth': 5, 'min_child_weight': 1},\n",
       "  {'max_depth': 5, 'min_child_weight': 3},\n",
       "  {'max_depth': 5, 'min_child_weight': 5},\n",
       "  {'max_depth': 7, 'min_child_weight': 1},\n",
       "  {'max_depth': 7, 'min_child_weight': 3},\n",
       "  {'max_depth': 7, 'min_child_weight': 5},\n",
       "  {'max_depth': 9, 'min_child_weight': 1},\n",
       "  {'max_depth': 9, 'min_child_weight': 3},\n",
       "  {'max_depth': 9, 'min_child_weight': 5}),\n",
       " 'rank_test_score': array([10,  9, 11,  1,  3,  6,  5,  2,  4, 12,  8,  7]),\n",
       " 'split0_test_score': array([-0.59542345, -0.5950706 , -0.59472178, -0.58310557, -0.58310533,\n",
       "        -0.58431803, -0.58411731, -0.58226497, -0.58221094, -0.59332021,\n",
       "        -0.59031767, -0.59008141]),\n",
       " 'split0_train_score': array([-0.57569263, -0.57661766, -0.57699247, -0.50853821, -0.513405  ,\n",
       "        -0.5188506 , -0.40509503, -0.42715968, -0.44039176, -0.28158973,\n",
       "        -0.32971623, -0.35677368]),\n",
       " 'split1_test_score': array([-0.59828358, -0.59765093, -0.59830697, -0.58714894, -0.58720018,\n",
       "        -0.58759295, -0.58918344, -0.58787471, -0.59021875, -0.60104223,\n",
       "        -0.59561471, -0.59470762]),\n",
       " 'split1_train_score': array([-0.57447548, -0.57526443, -0.57547798, -0.50652733, -0.51221488,\n",
       "        -0.51581781, -0.4040829 , -0.42624804, -0.4378755 , -0.27793942,\n",
       "        -0.32231255, -0.35279419]),\n",
       " 'split2_test_score': array([-0.59831815, -0.5985012 , -0.59826506, -0.58591037, -0.58816786,\n",
       "        -0.58788153, -0.58440312, -0.5867974 , -0.58768008, -0.5997458 ,\n",
       "        -0.59680517, -0.59444454]),\n",
       " 'split2_train_score': array([-0.5746644 , -0.57514501, -0.57534026, -0.50628355, -0.51269715,\n",
       "        -0.51682918, -0.40442883, -0.42546585, -0.43898629, -0.27508608,\n",
       "        -0.32286831, -0.35211017]),\n",
       " 'split3_test_score': array([-0.60073941, -0.60057825, -0.60085859, -0.58888468, -0.59009543,\n",
       "        -0.59018941, -0.59191571, -0.58807058, -0.58980508, -0.60661077,\n",
       "        -0.60255843, -0.59809808]),\n",
       " 'split3_train_score': array([-0.57367319, -0.57416848, -0.57450667, -0.50619658, -0.5115492 ,\n",
       "        -0.51514604, -0.405279  , -0.42453125, -0.43969196, -0.27608499,\n",
       "        -0.32382341, -0.35171245]),\n",
       " 'split4_test_score': array([-0.59936922, -0.5998796 , -0.6002126 , -0.58687228, -0.58729255,\n",
       "        -0.58751751, -0.58735858, -0.5885449 , -0.58692268, -0.60165276,\n",
       "        -0.59649081, -0.59763986]),\n",
       " 'split4_train_score': array([-0.57581789, -0.57654638, -0.57672755, -0.50634354, -0.51238983,\n",
       "        -0.51606529, -0.39915013, -0.42308085, -0.4379793 , -0.27570246,\n",
       "        -0.32274383, -0.35096815]),\n",
       " 'std_fit_time': array([  1.34286050e+00,   2.41265850e+01,   1.86517596e-01,\n",
       "          2.14092086e+00,   1.53126903e+00,   5.89184963e+00,\n",
       "          2.99300842e+00,   1.40887468e+00,   5.00864005e-01,\n",
       "          4.17511201e-01,   4.81990339e+00,   5.49777215e+02]),\n",
       " 'std_score_time': array([ 0.01304961,  0.00989216,  0.00719069,  0.00864679,  0.0182232 ,\n",
       "         0.02833219,  0.07318923,  0.17067846,  0.17626876,  0.29615184,\n",
       "         1.19328045,  0.45172095]),\n",
       " 'std_test_score': array([ 0.00174868,  0.00192719,  0.00213823,  0.00190002,  0.00228442,\n",
       "         0.00187155,  0.00294394,  0.00229526,  0.00286215,  0.00426812,\n",
       "         0.00389116,  0.00286893]),\n",
       " 'std_train_score': array([ 0.00080067,  0.00092594,  0.00092409,  0.00088684,  0.0006072 ,\n",
       "         0.00127378,  0.00227048,  0.00140714,  0.00097172,  0.00235536,\n",
       "         0.00275622,  0.00203856])}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.586384 using {'min_child_weight': 1, 'max_depth': 5}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEFCAYAAAASWssjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VOW9+PHPrJklCwECCfv+RCSgRQQVlWrlWperrVev\nF62/q9hqxdrS7brdq7Yqt16pikvd6lpvW23rdlFrS11xRUUCkgfCIiAkmezLZJvl98c5k0ySmUyG\nZDIhfN+vV17JnG2+c5I83/M855zvsYTDYYQQQojeWNMdgBBCiKFPkoUQQoiEJFkIIYRISJKFEEKI\nhCRZCCGESMie7gBSwedr6NclXrm5Hmpq/AMVzoCRuJIjcSVH4krOcIwrLy/LEm+e9CxisNtt6Q4h\nJokrORJXciSu5BxucUmyEEIIkZAkCyGEEAlJshBCCJGQJAshhBAJSbIQQgiRkCQLIYQQCUmyEEII\nkdCwvCnvYAUCQT58cxfjJoygYHIOLrcj3SEJIcSQIMkiSltrkK2bDrBpwz6sNgtTZoymcG4+E6fm\nYrVKJ0wIcfiSZBHF43Wy7IqF7N9dy4b3d7NT+9ipfXgzncyaMxZVVEDuKE+6wxRCiEEnyaIbj9fJ\ncUumM312Hr6yBrZuKqP0i3I++2Avn32wl/zx2ai5+cwoHIMzQ3afEOLwIK1dHBaLhTEF2YwpyOaE\nU6aza3slJZvK2Le7hrKv6ln/91KmzcqjcG4+4yaNwGKJW39LCCEOeSlLFkopK/AAMA9oBS7XWpdG\nzV8A/BqwAGXAxUBbrHWUUjOAJ4AwsBlYobUOpSr27uwOGzNnj2Xm7LE01LWwbXMZJcVlbNtSzrYt\n5WTluCgsykcV5ZOV4xqssIQQYtCk8qztuYBLa30ccC2wOjJDKWUBHgEu1VovBl4DJveyzq+BG7XW\nJ2Ikl3NSGHevsnJczD9hCsuuWMg5y45CzRlLs7+Nj9/dze9+8wEv/X4j27aUE2gPpitEIYQYcKlM\nFpEkgNb6A+CYqHmzgCpgpVLqLWCk1lr3ss584C3z51eBb6Qw7j6xWCyMmzSCU846gv939fEs+aYi\nf0IOX31Zy7qXt/Lkfe/x1mua8v31hMP9eryGEEKkXSrPWWQDdVGvg0opu9Y6AIwGjgeuBkqB/1NK\nbYi3DmDRWkda3AYgp7c3zs319Lume15eVlLLj5+Qy0nfmEWVr5HPP97L5xv28cXGA3yx8QCjx2Zy\n1IKJzJ0/gczs/g1TJRvXYJG4kiNxJUfiSk4q4kplsqgHoiO2mokCjF5FqdZ6K4BS6jWMXkTMdZRS\n0ecnsoDa3t64v0+vysvLwudrOOj1ixZM4Mj549m3u5qSTWXs2l7J3/9vK+vWbmXStFEUzs1n8oxR\n2GzJdez6G1eqSFzJkbiSI3Elpz9x9ZZkUpks1gNnA88qpRYBxVHzdgKZSqkZ5knvE4HfAjvirPOZ\nUmqJ1vpN4JvAGymMe0BYrRYmTRvFpGmjaGlup/SLCkqKD/Dljiq+3FGFy+1g1pFjKZybz6gxmekO\nVwghepXKZPE8cJpS6j2Mk9KXKqWWAZla64eVUsuB/zVPdr+ntV5rXkHVZR1zWz8BHlFKOYGtwJ9S\nGPeAc7kdzJk/njnzx1NV0UjJJuNKqk0b9rFpwz5Gj82kcG4+M2ePlRIjQoghyTIcT776fA39+lCD\n0b0MBkN8WVpFSXEZe3ZUEQ6D1WZh6kyjxMiEKSOxWrveuzEcu72pJHElR+JKznCMKy8vK+4NY3JT\nXprYbFamqTymqTz8ja1s21JOyaYydpT42FHiw5vlZNacfAqL8hkxUkqMCCHSS5LFEODJzOCohZOY\nd+xEKg40ULLpAKVbK/js/T189v4e8idkU1hUQPZid7pDFUIcpiRZDCEWi4Wx47IZOy6bE06dwc5t\nlZRsOsBXX9ZStq+e9etKmTZrNIVzCyiYmCMlRoQQg0aSxRBld9iYdeRYZh1plBjRxWVs/6ICvbkc\nvbmc7BEuVFE+ao6UGBFCpJ4ki0NAVo6LYxZP4fRz5vD5p3spKS5jZ4mPj9/Zzcfv7GbClFwK5+Yz\ndeZo7I7+3YwohBCxSLI4hFisFsZPzmX85FxOPG0mpSUVaLMS7r7dNTgzbMyYPZbConzGFGTJMJUQ\nYsBIsjhEOTPszJ43jtnzxlFT5UcXl7FtcxlffLafLz7bT+5oD4VFBcyaMxaP15nucIUQhzhJFsNA\n7igPi5ZM49iTprB3Vw0lm8rYXVrJ+2/s4IM3dzB5ulFiZNL05EuMCCEESLIYVqxWK5Onj2LydKPE\nyPYt5ZQUl7G7tIrdpVW4PGaJkSIpMSKESI4ki2HK5XZQdMwEio6ZQGV5IyXFB9i+pZxNH+9j08f7\nyMvPMkuMjCHDJSVGhBC9k2RxGBg9NpPFY2dy3Nens3t7Fbr4AHt2VvPO6w28t66UqbNGo4oKmDAl\nt0eJESGEAEkWhxWbzcr0wjymF+bR1NjKts3GMFXpVh+lW314szJQRcYwVU6ulBgRQnSSZHGY8mZm\ncPSiSRy1cCLl++sp2VRG6dYKPn1vD5++t4eCCTkUzs1nemEeDqf8mQhxuJNW4DBnsVjIH59D/vgc\nTvjGDHZpHyXFZXz1ZS0H9tXxzt+2M71wDIVz8ymYICVGhDhcSbIQHRwOG7Pm5DNrTj71tc3o4rIu\nXzm5brPEyNh+Px5WCHFokWQhYsoe4WbBiVM5ZvEUvvqyFl1cxk7t46O3d/HR27uYODWXwrkFTJk5\nKt2hCjGgwuEwoXCYUChMKITxczhMMBQmHAoTCkMoFCZotVJZ7e9YNhgKEw7TuW7HNsx1umzDmBYM\nhQiHYqxjvkes6cb7dE43ttkZ58KicRSOzx7w/SLJIkpjczu3PrmBhuY2ACxYsFotWCzGcI3FAlbz\nuzEvMt2CNWqZ6HldplssWMHcZoztxpgXvb7H7aS1tb3b9K5xWWNtN1YcHd/jfL5u67imjKBwQhZ1\nBxqp2lvL3l017N1Vg81hpWD6KEaMyyIz143Naul4z1jbscaIOVWf4XAWaVzCZmMS3eiFzYYt0mBF\nXvsDYSqrGgmbDVL0NkKhMMFw53a6bsNorLq+jsyncxsdsZjv2+W1uY1w9Gvju8Nhx9/S3vE62GU+\n3V53ft7oRr+jgY009FExhUJdG+BD/XFw5TXNFP7b0QO+XUkWURx2K/mjPGS1OmlrCxIOdx4phM0/\n7s7XYQJBCIdDHfNC3b73nG5MGw5cwGgsjGoPsq/Ex74SH37CVBKmCgikO0BTvOTXNdkkl4QSJ8Pu\n07vOczrtNLe0xzzyDEU1nqEYR5wdDWxUIxyMWj7y+nAT2b9Wq/ll7v+ury3YbWB12DrnmdON5ej2\nOs52zO8ej4O21kCP6R3b6TK9Zyydr2PEHr1OvG3Fma6m51Ff6x/4fSyPVe0p1Y9LTJSE4iWb3Fyv\neeTXdf3Y2+maoGJvt+e0jm50r9vsnB4Mhgg1tVOxs5qmyiYIAxZwjfTgGeslY4QbLPRImL1/hq7v\n0etnMI8Eu283HA5js9tobQvETtwhCBNru13n9Yg3FCZMnP3Wz6PS6IbGYrVgi2oELGaj0NFzM6d3\nfR2Z39lDtUU1QBZzenTvz2rt3IbH46S1NRC1zc7G02btfE+rGZslqqGyWS1dG+zIOlGfqXMbkdit\nvb5HZNro0ZnU1Pg7Pq+lx3uQlgsv5LGqIuU6jjRJ7g88b6QHSzCYoqgOXuSPs9nfxvYtFZQUH6Cq\noomWKj9uj4NZc8ZSWFTAyDxvWuIaTOGwkTC6J6hQ1Hh2Xl4W1VWNPRrydF9pNlQbv1E5bkJtQ6Wv\nevhKWbJQSlmBB4B5QCtwuda6NGr+SuBywGdOugLYDTwOTAPqgRVa6+1KqaOABzFGN7aZ2wqlKnZx\ncNweJ3MXTGDuggn4yhqMSrhbyvn8o318/tE+xhQYJUZmHDF8S4xYLMa5p95OmmS6HTRnyHGaOLSk\n8i/2XMCltT5OKbUIWA2cEzV/PnCJ1vqTyASl1NVAo9Z6kVJKAfcB/wTcBPxCa/2KUuoZ4Ezg5RTG\nLvopLz+LvPwso8RIaSUlm8rYu6uaigMNrF+3g6mzRlNYlM+EKblpP6IWQiSWymSxGHgNQGv9gVLq\nmG7z5wPXKaXygbVa61XAbOBVcx2tlDrCXPYzYKRSygJkAe0pjFsMIJvdyvTCMUwvHENjQyvbNpcZ\nJUa+qKD0iwoyszNQc/JRRfnk5LrTHa4QIo6UneBWSj0K/Flr/ar5eg8wTWsdMF/fBNyPMdz0PPAb\nYBywEGN4aiGwHnACF5jLVgB1wMla65Z47x0IBMN2uzxedKgKh8Ps213Dxo/3smXjV7S1GudhJk8f\nxVELJnLE3AKcMkwjRDqk5QR3PUYvIMIalSgswN1a6zrz9VrgaGAVcATwDkai+ERrHVRK3QOcqLXe\nopRagTGktSLeG9fU9O+ysaF6om84xeXKdLDo69OYf8JkdpolRr7cUcWXO6p45S/FTC/Mo7Aon/x+\nlBgZTvtrMEhcyRmOceXlZcWdl8pksR44G3jWPGdRHDUvG9hsDjM1AacAjwELgHVa65XmsNVkc/lq\njOQDsB84IYVxi0HkcNqMEiJFRomRErO0SMkm40tKjAgxNKQyWTwPnKaUeg+ja3OpUmoZkKm1flgp\ndT3wBsaVUuvMk9ejgV8qpW4AaoHl5rYuB/6glAoAbcB3Uxi3SJPsEW6OPXEqC8wSIyWbDrBzWyUf\nvb2Lj9/ZxYSpIyksymfqzNHY7PJ4WCEGk9yUF8Nw7F6mUirjam1pp3Srj5LiA1TsN94jw2Vn5uwx\nFM4tYPTYzLjDVIfj/uoPiSs5wzEuuSlPHLIyXA6OPHocRx49jurKJko2lbFtSxmbP93P5k/3MyrP\ni5qbz6wjx+L2ONMdrhDDliQLccgYOdrL8adMZ+HJU9m7s9o4KV5axXvrdvDBGzuZPGMUhUX5TJo+\nEqtVhqmEGEiSLMQhx2azMmXmaKbMHE2zv41tW8op2VTGrm2V7NpWidvrQM3J57iTp4PkDCEGhCQL\ncUhze5zMWzCRucdMoLK8kZJNB9j+RQUbP9zLxg/3MmZcFoVFBWaJEflzF+JgyX+PGBYsFktniZFT\nprN7exU7tY8dJT4q9jewfl0p02aNpnBuAeMnj5ASI0IkSZKFGHbsdhszjhjDcSdNZ9cOH3pzObq4\njO1fVLD9iwqysjOYVZRPYVE+2SOkxIgQfSHJQgxrmdku5h8/ma8dN4myfXWUFJexo8THJ+u/5JP1\nXzJu0ggKi/KZpvJwOKVEjBDxSLIQhwWLxULBxBEUTBzB4m/MYEeJUWJk/55a9u+p5Z2/bWfGEWMo\nLMpn7PhsGaYSohtJFuKw43DaKZxbQOHcAupqmtHFZejNZWz9/ABbPz/AiJGREiP5eLMy0h2uEEOC\nJAtxWMvJdXPsSVM5ZvEUvvqyhpLiMnZpHx++tYuP3t7FxGlGiZEpM6TEiDi8SbIQAuOZzxOnjmTi\n1JG0trSz/YsKdHEZe3ZUs2dHNRkuO7OOHIsqyicvP35lTiGGK0kWQnST4XIw52vjmfO18VT5Go3H\nw24up/iTryj+5CtGjfFSWFTAzCPHSIkRcdiQZCFEL0blZXL8KTNYePI09uyopqT4AHt2VLN+XSnv\nv7GDKTNHUVhUwMRpuVJiRAxrkiyE6AObzcrUWaOZOms0/qY2tm8pp6S4jJ26kp26Ek+mk1lHjqVw\nbj65o7zpDleIASfJQogkebxO5h07kbkLJuAra6BkU1mXEiNjx2VTODefGUeMkcfDimFD/pKFOEgW\ni4UxBdmMKcjm+FOns2tbJbq4jL27aijfX8/6v5cyTeWhivKlxIg45EmyEGIA2O02Zs4ey8zZY2ms\nb0FvLqdk0wG2bSln25ZysnJcqDnm1VS9POdYiKFKkoUQAyy6xMiBvZESIxVsWP8lG9Z/ycjRXrxZ\nTrJyXF2+snNcuL1O6YGIIUmShRApYrFYGDdpBOMmjeDE04wSI3pzObVVfqorm2KuY7NbycrO6Ewg\nI9xdEorb45BkItJCkoUQgyC6xEheXhb7v6qhoa6VhroW6uuaaahr6fJVW90cczt2u5VMsxfSvWci\nyUSkUsqShVLKCjwAzANagcu11qVR81cClwM+c9IVwG7gcWAaUA+s0FpvV0qNAR4BcgEbcInWekeq\nYhci1RxOOyPz7IzMi32ZbVtroEcCqY9OJlX+mOvZHdaYw1uRn11uSSbi4KSyZ3Eu4NJaH6eUWgSs\nBs6Jmj8fo9H/JDJBKXU10Ki1XqSUUsB9wD8BdwDPaK2fVUp9HSgEJFmIYcuZYWfUmExGjcmMOb+1\nJUBjfVQCqe2aVGoqe08mnQnEbQ53ufB6MgiHw5JMREypTBaLgdcAtNYfKKWO6TZ/PnCdUiofWKu1\nXgXMBl4119FKqSPMZU8ANiml/o7R+/hhCuMWYsjLcNnJcPWWTNrNYa7mLj2SyFe8ZOJw2mL2SCIJ\nxZlhl2RymLKEw+GUbFgp9SjwZ631q+brPcA0rXXAfH0TcD/GcNPzwG+AccBCjOGphcB6wAm0AN/T\nWj+ulPovwK61/q947x0IBMN2uzzIRohYwuEwLc3t1FY3U1fjp7baT21NM7VVfmrN122twZjrZrjs\njMj1kDPSzYiRHkbkmt/NL5fbMcifRgywuEcCqexZ1APRF5RboxKFBbhba11nvl4LHA2sAo4A3sFI\nFJ9orYNKqSrgJXM7LwO39fbGNTWxj5r6Ki8vC5+voV/bSAWJKzkSV+/sGVZG5WcyKt/onUTiCofD\ntLZ0njOpjxriaqhvoaqykfID9TG36cyI7pm4e5w/yXAl3+QMlf3V3XCMq7d7gFKZLNYDZwPPmucs\niqPmZQObzWGmJuAU4DFgAbBOa73SHLaabC7/LnAG8DRwErAlhXELcVizWCy43A5cbkfMcuyRZFJf\n2/Mqrvq6FupqmqmqiH1pcIbLHvMqrsiQl5RHGbpS+Zt5HjhNKfUeRtfmUqXUMiBTa/2wUup64A2M\nK6XWaa1fUUqNBn6plLoBqAWWm9v6CfCoUur7QB2wLIVxCyF6EZ1MxhRk95gfGebqchVXbdcruSrL\nG2NuOzqZZOe4yBrhYsKkkYQIkZ3jwuGUZJIuKTtnkU4+X0O/PtRw7F6mksSVnMM9rnA4TLO/Pe5l\nwQ11LQQDoZjrutz2rldxdeuhOJyDd65yOP4e8/Ky0nLOQggherBYLHi8TjxeJ2PHxe6ZNDe1dSSQ\nUCBM2f66jqRS7WvCVxa7Z+LyOGLesJid4yIzx4XDIRe+HCxJFkKIIcViseDJzMCTmUH++JweR8rh\ncBh/U1vXnknUMFdlRSMVB2IfWbs9jo7LgHucO8l2YZdkEpckCyHEIcViseDNzMBrJpPuwuEw/sa2\nmPeXNNS1UFkeP5l4vD0LPEaSS2Z2BofzJfmSLIQQw4rFYsGblYE3K4OCCT2TSSgUxt/YGvecia+s\ngfL9sS8N9mR2JpOxBdldy6tku7DZh++jdSVZCCEOK1arhcxsF5nZLgom9pwfSSbdS6lEXlfsr6f8\nq3pKv6josa43kkxG9LwsODPbhc126CYTSRZCCBElOpkQM5mEaGpow2qxsPfLmq7DXLXNlO+vp+yr\n2D0Tb1ZG3FIq3qyMIZ1MJFkMotbWVl5//VXOPvvcPq+zceOnZGZmMWPGzBRG1ncvvPAnqqqqWL78\niqTWe/HFv3Dmmf/Mpk0befHFP3PLLaviLvvmm+v43e+exGKB0077Jhdc8G/9DVuIAWO1GkNPeXlZ\neLMzeswPBkM0NcQf5ir/qo6yfXU91rNYzGSS7eronUQnlXQnE0kWg6i6uoqXX34hqWSxdu1LnHrq\n0iGTLA7W008/zumnn5lwuWAwyIMP3sejjz6N2+3m4ovPZ+nSbzJixIhBiFKI/rPZrGSPcJM9wh1z\nfnQy6VJKxSyncmBfHQd6SSbxnmWSmZ2B1Zq6ZHJYJotn/1HKxyU9xxsjbDYLwWBy9/UtKBzDBafM\n6HWZp556jN27d/HYYw+zc2cpdXXGH8SPfvQzpk+fwe2338K+fXtpbW3l/PMvZMqUaXz44fts21bC\nlCnTYtZteeWVl1m//m1aW1upqqrk/PP/jXfeeYtdu3awYsUPOfHEJfz5z3/krbfeoLm5mREjRnD7\n7Xfy0kvPs2nTRm655XZuvfUmZs+ew7e/fX7MuD//fCP33HMnWVnZ2Gw2jjxyDgB/+tMf+Nvf/orT\naeekk07l/PMv5LbbbiYcDlNRUU5zs58bb/wFxcUbqa6u4uabr+f88/+NvXv38pOfXENNTTUnnHAi\ny5dfweuvv0Zzs59zzvk2v/vdc9jtdmpqqgmFQjgch+WfqRimopPJ+Mk95weDIRrrW3tcxRV5SNb+\nvXWwN3Yyycx2cfSxkzhy/rgBj1v+CwfRJZdcxo4dpbS0tDB//rF861v/wt69e7j99ltYvXoNGzd+\nykMPPYHFYuGjjz6gsPAIFi48jlNPXUp+fn7c7fr9fu66637+/ve/8sc//i8PP/wEn332Cc8993tO\nOOEk6urquPvuB7Barfz4x1ezdesWzjvvAjZs+JDbbruZ9vb2uIkCYPXqVdx66x1MmjSZO+80ho92\n7drJunV/44EHHiUvL4uLL76EhQsXATB+/ARuvPEW3n//XR544B5+9au7eOKJ33LzzbezZUsxbW1t\nrFp1J6FQiPPOO5Ply69g6dLTO97Pbrfz1lv/4Ne//hXHHbcYlyv2EZoQw5HNZiUn101ObpyeSSBE\nY0PsUioNdS3UVMeuy9Vfh2WyuOCUGb32AlJ9G//OnaV8+ukG1q17HYCGhno8Hi/XXPMT7rjjNvz+\nJpYu/WaftzdzpgIgMzOLKVOmYrFYyMrKorW1DavVisPh4Oabb8DtdlNRUUEgEADgoov+nSuvvJTf\n/vZ3vW6/urqaSZOMQ6Cionns27eXnTt3UF5exg9/+H2cTjt1dXXs3bsXgK99bQEAc+bMY82aX/fY\n3rRp03E6nQDYbLH/BE8++RROPHEJt912M6+9tpYzz/znPu8PIYYzm91KTq6HnFxPzPmpar8SDnAp\npUYqpb5h/nydUuo5pdTsAY/kMGCxWAmHQ0yePIULLljGffc9zC9/+d8sXfpNKisr0Xorq1bdyR13\n3M1vfrOGQCCAxWIhHI5dJ6dzu/EfRlNaup23336TX/xiFStX/rxjW+3t7axZs5qf/ex6Vq/+b9rb\n2+NuIy8vj927dwGwdesXAEyaNJkpU6Zx770P8fTTT3PGGWcxfbpxXkXrrQAUF3/O1KnToz572Pw5\n/mdpamrk6qu/R1ubkejcbndKx2GFEH3Tl57F74GXjaeccj5wF/AgRqlwkYTc3Fza2wP4/X7eeONv\nvPTSX/D7m7jssu8xatQoqquruPLKy7BarVx44cXY7XZmz57Dgw/eR0HBePLy5ib9nhMmTMTtdvP9\n718GwKhRo6ms9PGb36zh+OMXc84536ay0seDD97LD37w45jb+NnPrufWW2/C6/Xi8XjIyspi5sxZ\nHHPMAq66ajmhUJCZM48gLy8PgA8+eI93332LUCjE9dffBMC8eUfx059ew2WXfS/me0SfszjttNNZ\nseK72O12pk+fmVQvSwiRGgmrziqlPtJaH6uUuhfYrrVeo5TaoLXu/pjUIUOqzg6u6Lhuu+1mTj11\nKYsWHZ/mqA6N/TWUSFzJGY5x9bfqrFUpNR84FzhZKXVUH9cTA+jOO/+b/fv30NYW6DJ99eo1ZGS4\n+r39srIybr2155Nqjz56ftL3VAghhp++NPr/AfwPcKfWeqdS6gNgZWrDEt399KfXpvRIJj8/n/vu\ne7jf27nhhpv7H4wQYshJeOZQa70O+KbW+h6l1Azgl8BbKY9MCCHEkNGXq6H+E3hEKTUJeBv4EfBQ\nqgMTQggxdPTlmsRzgO9iPPf6d1rr04CjUxqVEEKIIaUvycKmtW4FzgJeUUpZAW9qwxJCCDGU9OUE\n9zql1GbAjzEM9RbwUqKVzKTyADAPaAUu11qXRs1fCVwO+MxJVwC7gceBaUA9sEJrvT1qnWXAD7TW\nx/Uh7iFHqs72rersH//4DC+//GJH8cCf//x6Jk2a0p+whRD91JcT3D8FzgAWaa1DGI31f/Rh2+cC\nLrNhvxZY3W3+fOASrfUS80tjDHc1aq0XAT8A7ossrJQ6GlgO9HL/79AWqTqbjLVrX6Ky0pd4wSHu\n6acfJxgM9mlZrUu48cZbuO++h7nvvoclUQgxBCTsWSil8oA7gVOUUnbgDaXUlVrr8gSrLgZeA9Ba\nf6CU6n4T33zgOqVUPrBWa70KmA28aq6jlVJHmDGMAm7HOLn+SJ8/XRx/Kf0/PqsojjvfZrUQDCV3\nX9/RY4r49oyzel1Gqs72reqs1lv53e8ep6qqiuOPX8x3vnNpUr8LIcTA68sw1EPAexhDRlbge8Bv\nMc5h9CYbiK6jG1RK2bXWkbvK/gDcjzHc9LxS6ixgI3CWUuoFYCEwXillM9/vx0BzXz5Ubq6n1wer\ne75yYrP23kFJNL/HNt3OmI15tB/96Afs2bMLqzXEySefyLJly9i9ezfXXXcdjzzyCMXFG3n22WcB\nWL9+PSeeeCwnn3wSZ5xxBkVFxjBU9/fIynIRCLTx5JOPs3btWp544gmeffZZPvzwQ5566inOPfdM\nAoEWnnnmaaxWK8uXL+fAgV1ceeVyrrrqKlavvg2bDa644rK4cd9zzx2sWbOGqVOnctNNN+H1ZlBX\nV87bb/+D5577IwCXXnopp59+Ki6XgwkTJnDPPb/mrbfe4tFH7+fBBx/k6acf5/7772Xjxo2EQgEe\nffQhgsEgS5Ys4dprf8pFF3Umqn/+57NZtmwZmZmZXH311WzevIGvf/3rSf0+IhL9TtJF4kqOxJWc\nVMTVl2QxTWv97ajXdyilvtOH9eqB6IitkUShlLIAd2ut68zXazGusFoFHAG8A6wHPsHogcwEfgO4\ngNlKqbu11j+K98Y1Nf5eAzt9/FJOH7807vyDvfkt0TrV1U20twfZvPkL3n33PV588WUAamtraG4O\ns2LFSn7Cy0guAAAgAElEQVT+8+s6qs76fA20tLRTV9eMz9cQM66GhhYmT56Oz9dAKGRn/PhJVFY2\nEgrZaWjwU1XVRFtbiBUrrsHtdrNv334qK+vx+Ro4//yLO6rO9hZ7RYWPzMzR+HwNzJw5m3379rJh\nwyb27fuKZcsuxum0U1VVw6ZNJbS0tFNYOA+fr4GJE2dSWroDn6+BYDCEz9dAba2fSZOmUlfXCoDV\nauvy3uFwmDPPPI9g0EFdXSvz5y9iw4aNzJmTfHWZ4ViOIZUkruQMx7h6SzJ9uRoqrJTqeBKteb9F\n/BKlndZjnOtAKbUIiB73yQY2K6UyzcRxCkZiWACs01ovBp4DdmqtP9JaH6m1XgJcCHzRW6IYyqTq\nbF+qzjZxySX/it/vJxwO8+mnH6NUYa+fXwiRen3pWfwn8L5S6kOMk8sLMa5cSuR54DSl1Hvmepea\nVzNlaq0fVkpdD7yBcaXUOq31K0qp0cAvlVI3ALUYJ7SHDak627eqs9/73lVcc82VOBwOjjnmWI47\nbnHSn1sIMbASVp2FjpPcx2L0RD7UWsd/JukQIFVnB5dUnU2OxJUciSs56aw6i9baB6yNvFZKFWut\niw4qGnFQpOqsECKdDrbU+JSBDEIkJlVnhRDpdLDPq+zXMI8QQohDizzcWAghREJxh6GUUiFi9yAs\ncaYLIYQYpuImC6219DqEEEIAMgw1qFpbW5MuJLhx46eUlm5PvOAgeeGFP/Hb3yb/7KsXX/wLgUCA\nTz/dwE03XRd3uaqqSq6++nsdX6efvoQXXvhTf0IWQgwASRaDSKrOJq46O2rU6I5qs1deeTWzZhVy\n9tnfGoQIhRC9OdhLZw9pvuf+QMOGj+PO/9JmJRjsvcRGd1nHLCDv/At7XUaqzvat6iwYNaLuuut/\nuOmmX2KzxS8KKYQYHH0pUd79Tq0wRvXXrVrrtTFWEXFccsll7NhRSktLC/PnH8u3vvUv7N27h9tv\nv4XVq9ewceOnPPTQE1gsFj766AMKC49g4cLjOPXUpeTn58fdrt/v56677ufvf/8rf/zj//Lww0/w\n2Wef8Nxzv+eEE06irq6Ou+9+AKvVyo9/fDVbt27hvPMuYMOGD7nttptpb2+PmygAVq9exa233sGk\nSZO5807joUW7du1k3bq/8cADj5KXl8XFF1/CwoWLABg/fgI33ngL77//Lg88cA+/+tVdPPHEb7n5\n5tvZsqWYtrY2Vq26k1AoxHnnncny5VewdOnpXd5z/fq3mTp1mjzLQoghoi89ixkYVV9/b74+D6Oi\n7GKl1Mla65+nKrhUyTv/wl57Aam+jX/nzlI+/XQD69a9DkBDQz0ej5drrvkJd9xxW0fV2b6aOVMB\nkJmZxZQpU7FYLGRlZdHa2obVasXhcHDzzTfgdrupqKggEDDuAr/oon/vqDrbm+rqaiZNmgxAUdE8\n9u3by86dOygvL+OHP/w+Tqeduro69u7dC8DXvrYAgDlz5rFmza97bG/atOk4nU4AbLbYf4J//eur\nnJ+gpyaEGDx9SRYKOMl8DjdKqQeBt7TWxymlPgcOuWSRLtFVZ5cunc3SpadTU1PNyy+/0KXqbGtr\nK+eddyb/9E9nDFjV2UceeZKWlhaWL78Y6Fl19v77H8HhcMTcRqTq7JQpU9m69QuysrI6qs6uXr2G\nMWOyue++B5k+fSZvvrkOrbcyb95RB1V1NqKkZCtFRfMSLyiEGBR9SRa55nKt5msnkGn+LCfIkyBV\nZ/tWdbampgav19trEhRCDK6EVWeVUtcA3wf+D7AB3wTuxUgaC7TWF6U6yGRJ1dnBJVVnkyNxJUfi\nSk7aqs5qrdcopd4AvgEEgH/RWm9RSs0EHjioiETSpOqsECKd+tKzsABXYiQLG8YDi+7VWid3bekg\nkp7F4JK4kiNxJUfiSk46n2dxB8bVUI9hPvEOmAocko82FUIIkby+JIulwNGRnoRSai1dn6cthBBi\nmOvL1Ux2uiYVO5C4boMQQohhoy89i2eAN5VSkZvy/o3OG/SEEEIcBvpyNdTtSqnPgFMweiK39aXM\nh1LKinG11DyMezQu11qXRs1fCVwORKrkXQHsBh4HpmHcJb5Ca71dKXUUxuW6QXNbl2ity/v6IYeK\n1tZWXn/9Vc4++9w+r7Nx46dkZmYxY8bMFEbWdy+88CeqqqqSvkLqxRf/wpln/jObNm3kxRf/zC23\nrIq77GuvreX3v38arzeTM844i7PO6vv+EkKkRp9uqtNav6q1/pnW+ida67VKqb5cMnsu4NJaHwdc\nC6zuNn8+RqO/xPzSwHeBRq31IuAHwH3msvcAP9BaLwH+AvxHX+IeaqTqbOLRy9raWh599EHuvfch\n7rvvYV5//TUOHNg/CBEKIXpzsFVnLwauSrDMYuA1AK31B0qpY7rNnw9cp5TKB9ZqrVcBs4FXzXW0\nUuoIc9kLtdYHomJuOci4AXjvHzvYWVIRd77VZiWUZNXZaYVjOP6U6b0uI1VnE1ednTlzFjNmzCQ7\nOweAwsLZbNlSTEHBuKR+H0KIgXWwyaIvdRiygbqo10GllF1rHbmr7A/A/RjDTc8rpc4CNgJnKaVe\nABYC45VStkiiUEodD1wNnNTbG+fmerDb45e19nicWG29d6oSzY+1zViNebQf/egH7NmzC6s1xMkn\nn8iyZcvYvXs31113HY888gjFxRt59tlnAVi/fj0nnngsJ598EmeccQZFRcYwVPf3yMpyEQi08eST\nj7N27VqeeOIJnn32WT788EOeeuopzj33TAKBFp555mmsVivLly/nwIFdXHnlcq666ipWr74Nmw2u\nuOKyuHHfc88drFmzhqlTp3LTTTfh9WZQV1fO22//g+ee+yMAl156Kaeffioul4MJEyZwzz2/5q23\n3uLRR+/nwQcf5OmnH+f+++9l48aNhEIBHn30IYLBIEuWLOHaa3/KRRcZiaquro5bb92NxdKK1+vl\n888/YfbsWQn3bTwHu16qSVzJkbiSk4q4DjZZ9OWmt3ogOmJrJFGYN/rdrbWuM1+vBY4GVgFHAO8A\n64FPtNZBc5l/BW4AztRa9zouU1Pj7zWwoxZN5KhFE+POP9ibWhKtU13dRHt7kM2bv+Ddd9/jxRdf\nBqC2tobm5jArVqzk5z+/rqPqrM/XQEtLO3V1zfh8DTHjamhoYfLk6fh8DYRCdsaPn0RlZSOhkJ2G\nBj9VVU20tYVYseIa3G43+/btp7KyHp+vgfPPv7ij6mxvsVdU+MjMHI3P18DMmbPZt28vGzZsYt++\nr1i27GKcTjtVVTVs2lRCS0s7hYXz8PkamDhxJqWlO/D5GggGQ/h8DdTW+pk0aSp1dUapMavV1u29\nrVx11Y+48sqryMnJYdq0WdhsroP6fQzHm6ZSSeJKznCMq7ckEzdZmCU+YiUFC+Duw/uuB84GnlVK\nLaLrvRnZwGZzmKkJ4+T5Y8ACYJ3WeqU5bDXZjOVijBPgS7TW1X147yFJqs4mrjobCATYtq2EBx54\nlPb2dlauXMEVV6zo9fMLIVKvt57Fzf3c9vPAaUqp9zDv/FZKLQMytdYPK6Wuxygd0oqRIF5RSo0G\nfqmUugGoBZYrpWzAGmAP8BelFBgl0m/qZ3yDTqrO9q3qLMBll12E05nBhRdexIgRI5L+3EKIgZWw\nNtShSGpDDS6pOpsciSs5Eldy0lkbSgwBUnVWCJFO0rOIYTgeMaSSxJUciSs5EldyUtWzkCfdCSGE\nSEiShRBCiIQkWQghhEhIkoUQQoiEJFkIIYRISJKFEEKIhCRZCCGESEiShRBCiIQkWQghhEhIkoUQ\nQoiEJFkIIYRISJKFEEKIhCRZCCGESEiShRBCiIQkWQghhEhIkoUQQoiEJFkIIYRISJKFEEKIhCRZ\nCCGESMieqg0rpazAA8A8oBW4XGtdGjV/JXA54DMnXQHsBh4HpgH1wAqt9Xal1AzgCSAMbDanh1IV\nuxBCiK5S2bM4F3BprY8DrgVWd5s/H7hEa73E/NLAd4FGrfUi4AfAfeayvwZu1FqfCFiAc1IYtxBC\niG5S1rMAFgOvAWitP1BKHdNt/nzgOqVUPrBWa70KmA28aq6jlVJHRC37lvnzq8BS4Pl4b5yb68Fu\nt/Ur+Ly8rH6tnyoSV3IkruRIXMk5nOJKZbLIBuqiXgeVUnatdcB8/QfgfozhpueVUmcBG4GzlFIv\nAAuB8UopG2DRWofN9RqAnN7euKbG36/A8/Ky8Pka+rWNVJC4kiNxJUfiSs5wjKu3JJPKYah6IPqd\nrZFEoZSyAHdrrSu11m3AWuBo4DFzvXeAbwGfaK2DQPT5iSygNoVxCyGE6CaVyWI9cAaAUmoRUBw1\nLxvYrJTKNBPHKcAnwAJgndZ6MfAcsNNc/jOl1BLz529iJBMhhBCDJJXDUM8Dpyml3sM4KX2pUmoZ\nkKm1flgpdT3wBsaVUuu01q8opUYDv1RK3YDRe1hubusnwCNKKSewFfhTCuMWQgjRTcqShXlp65Xd\nJpdEzX8aeLrbOpXAN2JsaxtwcgrCFEII0QdyU54QQoiEJFkIIYRISJKFEEKIhCRZCCGESEiShRBC\niIQkWQghhEhIkoUQQoiEJFkIIYRISJKFEEKIhCRZCCGESEiShRBCiIQkWQghhEhIkoUQQoiEJFkI\nIYRISJKFEEKIhCRZCCGESEiShRBCiIQkWQghhEhIkoUQQoiEJFkIIYRIyJ6qDSulrMADwDygFbhc\na10aNX8lcDngMyddAewEngSmAEHgu1rrEqXUUcCDQADYZm4rlKrYhRBCdJXKnsW5gEtrfRxwLbC6\n2/z5wCVa6yXmlwbOAOxa6+OBXwC3mcveBPxCa70YyADOTGHcQgghuklZzwJYDLwGoLX+QCl1TLf5\n84HrlFL5wFqt9SqMXoPd7JVkA+3msp8BI5VSFiAranpMubke7HZbv4LPy8vq1/qpInElR+JKjsSV\nnMMprlQmi2ygLup1UCll11oHzNd/AO4H6oHnlVJnAZ9jDEGVAKOBs8xlt5vL3mhu883e3rimxt+v\nwPPysvD5Gvq1jVSQuJIjcSVH4krOcIyrtySTymGoeoxeQMd7RRKF2UO4W2tdqbVuA9YCRwMrgb9q\nrWdhnOt4UinlAu4BTtRaFwJP0XNISwghRAqlMlmsxzgHgVJqEVAcNS8b2KyUyjQTxynAJ0ANnb2R\nasAB2Myf683p+4HcFMYthBCim1QOQz0PnKaUeg+wAJcqpZYBmVrrh5VS1wNvYFwptU5r/YpS6m3g\nMaXUO4ATuF5r3aSUuhz4g1IqALQB301h3EIIIbpJWbIwL229stvkkqj5TwNPd1unEbggxrbeBU5I\nQZhCCCH6IJU9CyGEEHGEwiECoQDtoQDtoXYCoSCBUDvtoUDU9ACBUHu315H57d1eG9OOmVREUVbR\ngMcryaKb9poamtsbCLSEsbk9WOyyi4QYbkLhUFQjHUii0Q52NNLOA1bqG/1xthNpxNvjNvKhcGru\nK24O+ymaI8kipQK1tez6+Y/ZFQ53TLNkZGDzeLC6PVg9no6fbV7jtdVtTvN4sHm8xjSPB5u5vMUq\nFVWEiBYMBY3GM2w2nkGzse3xOkgg2I6rwU5NfaOxTtS8jiProLGtjka7y+vYR+KpaqijWbDgsDlw\nWOzYrXYcVjsupwuHxYbd6sBh7ZxufHdgt9rM73YcFhuOIDjbQjjawjjaAjjagthb27G1BrC2tmNr\nbcfa0oalpRXMr7HH5MOcgf88kiyi2LKzGf2t87DW1+CvqSPk9xP0+wn5mwjU1RI6sB+iEklfWF0u\nM4F4O5OKmUg6kk9UEuqc5sXqckmyEQMmHA6bR9Rdj5AjjWkVTnzVdb0eIQeipkVvp+v0OMuHB6+h\ntlqs3RphO267K6px7myYI8tF5nU21nbsNnN5i/nd5uh4nTcqm8b6NmN5ix2HzdyOuaw1DKHmZqMN\nafYb7UlzMyHztTE98rqeoLlMqLlzOUK976swRl2kzg9upW3iRLwp2KeSLKJYrFZGnnFW3JtawuEw\n4dYW45fc5O/85UaSSrOfYFOT+UcRNa/ZT6CqkrZ9zUkGZMHqdnckmLKcLEIOV9dEE+nluHsmH0uG\nC4vFMkB7R/RXOBymPdROfUsDNS21XRrdWGPP0cMh3Y+QezbO8aZ3bbTDJHewczBsFluXBthpc+Bx\nuDsa6dhH1Pa400fmZNLcFIhav/PIvGtD7zAbaxs2a/8qOBj/662djXvH/3hjR4OeQYD2qjpCzX5a\n/c00N3dNAuHW1qTfNzKSYc/Oxjo2v3PkoqMdcHcd0ej22uJ0MmZMdkpuFpRkkQSLxYLF5cbqcsPI\nUUmvHw6FCLU0R/VYOr/3SDD+pi5JqN1XQevePcm9odXapScTSSRdEk2Xad4u8yxOpySbOAKhAE3t\nzTS1N9HU7scf8NPU3vWrx7SAn0AokHjj/WA3hzgiR80Ztgy8Dm+3o+bYjW1Oppe2lmCcxrt7Ix+/\n0bZaBrY3fDB3JIeDwSSO6pu7HtWb0xId1ffQ8f/mxp6d0/m/19Gguzv/33okAY8xkjCEz5EO3ciG\nIYvVis3jxebx4jiI9UeP9FC+p6IzwTSbSSWql9NlXlNTxx9/4EAt4ba25N7QZusYEuvxRx6VfBg7\niqagxezleDt6OVbHwXzKwRUKh/BHGv1AVOPf0cAb0yLLNJpJoDXY933ptrvxOjyMzyjA43CT48kk\n2E6Xxrf7EXLsoZFYY93Ry9r61VAPlfIV3Y/q6yuhcX9lVOMe1dg3+wn6je8DdlSfk4M1vyDhUf3I\nglHUt9HlqH44H1xJsjiEWGw2bJmZ2DIzD2r9cCBgJJUmf9cjrm69mY5/uqjpgapKwoHYR8UV8eJ1\nODqPmswk0uMcjTdOLyfJK9HC4TDNgZaOo/nGdj92f4gDVdU0dRzhRxp9f8e05kDfhwYzbE48dg9j\n3KPxOrx4HG68Di9eMxn0nGa87t6AD5VGOVXCgYB5tB77SD7YHPnbM4/oYyyX9FG9zWYcubs9XY/q\no4/o4wzhRI7+Lbbkhq6y87JoHca/x+4kWRxGLHY79qxsyMo+qPVD7W1dx2/N7x5bkPqKmo6LAboM\nsTX7CTY10uargGAw8ZtEczoJuzMIZTgJuhy0O220ZdhodVhodoTx28M02YM02ALU29ppdkCr00qr\nw0Kbw0LYGvsoz26147V7yM3IYUJmAV6HB4/dYzb4nV+d07wdY+7DXcdRfZzGvbNB90cN8zR3GUZN\nugcLWDJc2Dxu7DkjsOYXdLngI3NUDq3YuzbuXZLA8D+qHwqG/1+/GDBWhxNrjhNyRnSZnpeXhT3G\nEVZ7KNBxNN/Y1ojfX09zYw0tDXW0NTXS3lRPsKmJYLOfsL8FS0sL1pY2HG0hXG0hnG0hMtqbyKht\nJKM9jDNMUld5hDOMZGMxGxW7JxOHNwuH19v16jSLB6vDg9UZNYx2iF6J1nFU32WYpvuRfPcxfOP7\nztZmAk2Jr8DpIfqoPr/3o/roJBBZJ9FR/XDviR0qJFmIhIKhIP5Ac5cTt41R4/rBL9upaqjrOtbf\n3kRbqNfHjoDb+LJgwW134XWMxBM5srd78TqM4R2Pw0OmzY037MDdDq52CxntIewtAcKRhjG6R2M2\nhta2FlrrGwjVNhA6UEEbRmGxPom+Ei1y4r9Plzwby1syMpI+0k33Ub0zNxf7mPyeDXr3IRs5qj8s\nSbI4jITCIVoCLR0naRNevWNOaw609Pk9XLYMPA4PY71j8EYN7XQmgZ7TPPae4/oDIfqItE9Xopm9\nnO6XQ7eVlxNu7fs+AOJfieb2UOu00lRT3+WoPjJ2fzBH9ZGjc3vBiC6NeNeTsu6ER/VyBC96I8ni\nEBQOh2kNthqXbga6Xb0Ta5qZBPztzX2+zt5hteN1eBnpysVjnqz1Rk7edhvjnzgmj9bGMB67G/sQ\nHdfv75VoXS7FjHNPTaSH0+V8TYwr0eqjtmsc1XuMsfqCcb1cRx/V2Ls92DxuOaoXg2po/mcfRtqD\n7VFX6/i7NPKNgaird9r9tIZbqGtpwN/eTDDct5PFVosVr8NDliOTfM+YziP8qKN8j8NDZuRErpkY\nnLa+N6l5OVn42ob3EWl/r0QLtbd39CJGjcmm1h86qCtwhEgXSRYDJBgK0hQwGvrG6EY/ViKImtae\naFzfZMGC1+nBY3Mz2jWyx7BOvCTgsiU/di4GntXhMO47yc7GlZeFTYZ7xCFGkkU3oXCIxtYmKvy+\nbmP3nTdsxRrjbwn2/SYgl82F1+GmwDumy9F812Gezp+9Dg9uu4uxY3JkTFkIkRaSLKI0tjVx64er\naWhv7NPyTqsDj8PDKPfIXk/mdh/m6W/dGiGEGGySLKJk2JyokTOw2MPYQ864V+9ETvAmM64vhBCH\nMkkWURw2B5ceuUwuIRRCiG4OvVtUhRBCDLqU9SyUUlbgAWAe0ApcrrUujZq/Ergc8JmTrgB2Ak8C\nUzCe6fFdrXWJUmoM8AiQC9iAS7TWO1IVuxBCiK5S2bM4F3BprY8DrgVWd5s/H6PRX2J+aeAMwK61\nPh74BXCbuewdwDNa65OAG4HCFMYthBCim1Ses1gMvAagtf5AKXVMt/nzgeuUUvnAWq31KmAbYDd7\nJdlA5CaEE4BNSqm/A7uBH/b2xrm5Huz2/l1xlJeX1a/1U0XiSo7ElRyJKzmHU1ypTBbZQF3U66BS\nyq61jjwU4Q/A/RjVD55XSp0FfI4xBFUCjAbOMpedAtRorb+hlPov4D+A/4r3xjU1/n4FPlRPcEtc\nyZG4kiNxJWc4xtVbkknlMFQ9EP3O1kiiUEpZgLu11pVa6zZgLXA0sBL4q9Z6Fsa5jieVUi6gCnjJ\n3M7LQPdeihBCiBRKZbJYj3EOAqXUIqA4al42sFkplWkmjlOAT4AaOnsj1YAD44T2u5FtAScBW1IY\ntxBCiG5SmSyeB1qUUu8BdwErlVLLlFLf01rXAdcDbwDvAFu01q+Yy31NKfUO8A/geq11E/AT4BJz\nW6cDt6cwbiGEEN1YwuG+lawWQghx+JKb8oQQQiQkyUIIIURCkiyEEEIkJMlCCCFEQpIshBBCJCTJ\nQgghREKSLIQQQiR02D/8SCm1EPiV1npJt+lnY9SfCgCPaa0fGSJx9SjtblbsTXU8DuAxjDpdGcCt\nWuuXouanZX/1Ia507S8bRll9BYSBK7XWm6Pmp2t/JYorLfsr6v3HYFRzOE1rXRI1Pd3/j/HiStv+\nUkp9ilFWCWCX1vrSqHkDvr8O62ShlPo58B2gqdt0B8bd5AvMeeuVUi9prcvTGZcpUtr9k8GIJcrF\nQJXW+jtKqZHARsx6XWneX3HjMqVrf50NoLU+QSm1BKPc/jmQ9v0VNy5TuvZXZL88BDTHmJ7O/8eY\ncZnSsr/MmnmW7geT5ryU7K/DfRhqB/DtGNOPAEq11jVmocN3MWpSpTsu6Czt/q5S6rpBjOk54D/N\nny0YRywR6dxfvcUFadpfWusXgO+ZLycDtVGz07a/EsQF6fv7ArgTeBDY3216uv8f48UF6dtf8wCP\nUup1pdQ/zPp7ESnZX4d1stBa/5nOZ2ZE615evQHIGZSg6DUuMEq7X4lRfHGxWdp9MGJq1Fo3KKWy\ngD9hPIQqIm37K0FckKb9ZcYWUEo9CdwLPBM1K91/X/HigjTtL6XUvwM+rfVfY8xO2/5KEBek7+/L\nj5HE/sl8/2eUUpGRopTsr8M6WfSie3n1LHoegQ26Xkq7D9b7T8Qo/vi01vp/o2aldX/Fiyvd+wtA\na/3/gFnAI0oprzk57X9fseJK8/66DDhNKfUmcBTwlPlgNEjv/oobV5r31zbgd1rrsNZ6G8ZjHArM\neSnZX4f1OYtebAVmmmPgjRhduDvTGxLQWdr9CIyxyFMwTu6mnFJqLPA6cLXWel232WnbXwniSuf+\n+g4wwXwCpB8ImV+Q3v3VW1xp21/mI5MjMb6JceK9zJyUtv2VIK607S+MJFYEXKWUGmfGcsCcl5L9\nJckiilJqGZCptX5YKfVj4K8Yva/HtNZfDZG4IqXdW4F1Zmn3wXA9kAv8p1Iqco7gEcCb5v2VKK50\n7a+/AI8rpd7GeC7Lj4BvKaXS/feVKK507a8e5P+xV78FnlBKvYtxVdtlwAWp/PuSEuVCCCESknMW\nQgghEpJkIYQQIiFJFkIIIRKSZCGEECIhSRZCCCESkmQhRBoppZ4w7xI+mHVvUUqdaP78plnrSYiU\nkGQhxKHrZMCW7iDE4UHusxACMI/Kb8AoRjgdo85UHXCuOe0M4HyMasBejLue/xXjDtlPMBruHcAG\n4Dqt9do472MBVgNnYRSmswG/1Vo/oZS6BOMmOau5zRVa6xallA/4P4yidQ3ARRh35T4AlAHfwqjz\ntA+jiFwu8EOt9csDs3eEkJ6FENEWApcCRwLfxyggdwywCbgQI3Es0VrPAV4ArtJa7wX+A/gNcBPw\nXrxEYToPo37QkRjJZwaAUupI4LvA8Vrro4AK4KfmOqOBN7XWczEK163RWj+FkZgu11oXm8vVaq3n\nA9dgPMtAiAEjyUKITpu11nu11n6gEojUmvoS42h9GXChUmoVxnMhMgG01o9jPOtgGfCTBO+xBPiL\n1rpda+0DIuUhvg7MBD5QSm3EeMZEoTmvBXjK/PlJjBpEsbxgft+CkWCEGDBSG0qITm3dXkc/G2Mi\n8D5wH/AqxvDP0dDxIJqJGP9PE4DenpQWputBWuQ9bMCzWutrzG1m0vn/GdJaR8aLrfR8Zkf3bYUx\nhs6EGDDSsxCibxZgPFDmLuBD4Jt0nlz+JfAPYCVGkb7e/q/+DpyvlMpQSuUCp5vT38Qo6DfGPK/x\nG4zzF2A85OZs8+dLMZIVGMlBDvjEoJBkIUTfvA5YlVJfAB8Au4GpSqnjMM493KC1/hNQTee5hh60\n1i9iJIbNGI9//cKc/jlwC0bS2YLxv/nfUauer5TahPGwm0gSeQ14UCl1/MB8RCHik6uhhBjilFJh\nrS//0mQAAABJSURBVLUMK4m0ki6sEAPMvFHu3jizz9Bax3qWsxBDmvQshBBCJCTnLIQQQiQkyUII\nIURCkiyEEEIkJMlCCCFEQpIshBBCJPT/AXyJiByCHZjsAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x273f8e9eb38>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_1.best_score_, gsearch2_1.best_params_))\n",
    "test_means = gsearch2_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch2_1.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(max_depth), len(min_child_weight))\n",
    "train_scores = np.array(train_means).reshape(len(max_depth), len(min_child_weight))\n",
    "\n",
    "for i, value in enumerate(max_depth):\n",
    "    pyplot.plot(min_child_weight, -test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [4, 5, 6], 'min_child_weight': [0.5, 1, 1.5]}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "max_depth = [4, 5, 6]\n",
    "min_child_weight = [0.5, 1, 1.5]\n",
    "param_test2_1 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda2\\envs\\python3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:667: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59104, std: 0.00162, params: {'min_child_weight': 0.5, 'max_depth': 4},\n",
       "  mean: -0.59176, std: 0.00179, params: {'min_child_weight': 1, 'max_depth': 4},\n",
       "  mean: -0.59160, std: 0.00169, params: {'min_child_weight': 1.5, 'max_depth': 4},\n",
       "  mean: -0.58683, std: 0.00221, params: {'min_child_weight': 0.5, 'max_depth': 5},\n",
       "  mean: -0.58638, std: 0.00190, params: {'min_child_weight': 1, 'max_depth': 5},\n",
       "  mean: -0.58684, std: 0.00169, params: {'min_child_weight': 1.5, 'max_depth': 5},\n",
       "  mean: -0.58516, std: 0.00297, params: {'min_child_weight': 0.5, 'max_depth': 6},\n",
       "  mean: -0.58524, std: 0.00325, params: {'min_child_weight': 1, 'max_depth': 6},\n",
       "  mean: -0.58575, std: 0.00296, params: {'min_child_weight': 1.5, 'max_depth': 6}],\n",
       " {'max_depth': 6, 'min_child_weight': 0.5},\n",
       " -0.58515689713818275)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=236,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_test2_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_1.grid_scores_, gsearch2_1.best_params_,     gsearch2_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'min_child_weight': [0.3, 0.5, 0.7]}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min_child_weight = [0.3, 0.5, 0.7]\n",
    "param_test2_3 = dict(min_child_weight=min_child_weight)\n",
    "param_test2_3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=3, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
       "       colsample_bytree=0.8, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=6, min_child_weight=1, missing=None, n_estimators=236,\n",
       "       n_jobs=1, nthread=4, objective='multi:softprob', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=3, silent=True,\n",
       "       subsample=0.3),\n",
       "       fit_params={}, iid=True, n_jobs=-1,\n",
       "       param_grid={'min_child_weight': [0.3, 0.5, 0.7]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_3 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=236,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=6,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        nthread=4,\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_3 = GridSearchCV(xgb2_3, param_grid = param_test2_3, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_3.fit(X_train , y_train)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda2\\envs\\python3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:667: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58569, std: 0.00299, params: {'min_child_weight': 0.3},\n",
       "  mean: -0.58516, std: 0.00297, params: {'min_child_weight': 0.5},\n",
       "  mean: -0.58542, std: 0.00281, params: {'min_child_weight': 0.7}],\n",
       " {'min_child_weight': 0.5},\n",
       " -0.58515689713818275)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_3.grid_scores_, gsearch2_3.best_params_, gsearch2_3.best_score_"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
